package etl

import org.apache.spark.rdd.RDD
import org.apache.spark.sql.types._
import org.apache.spark.sql.{DataFrame, Row, SQLContext}
import org.apache.spark.{SparkConf, SparkContext}

/**
  * Created by 王康 on 2018/6/29.
  */
object Bz2Parquet {
  def main(args: Array[String]) {
    val conf: SparkConf = new SparkConf().setAppName("Bz2Parquet").setMaster("local[4]")
    //设置kryo 序列化 模式
    conf.set("spark.serializer", "org.apache.spark.serializer.KryoSerializer")
    val ssc: SparkContext = new SparkContext(conf)
    val sqlcontext: SQLContext = new SQLContext(ssc)
    //在sqlcontext 上设置压缩文件格式
    sqlcontext.setConf("spark.sql.parquet.compression.codec", "snappy")
    //读取本地文件
     val file: RDD[String] = ssc.textFile("E:\\大数据资料\\project2\\dmpbig.FINISH")
    //构建一个schema信息  不然的话 写进去的是一个 value 所有字段 都在一个value中
    val schema: StructType = StructType(List(
      StructField("sessionid", StringType),
      StructField("advertisersid", IntegerType),
      StructField("adorderid", IntegerType),
      StructField("adcreativeid", IntegerType),
      StructField("adplatformproviderid", IntegerType),
      StructField("sdkversion", StringType),
      StructField("adplatformkey", StringType),
      StructField("putinmodeltype", IntegerType),
      StructField("requestmode", IntegerType),
      StructField("adprice", DoubleType),
      StructField("adppprice", DoubleType),
      StructField("requestdate", StringType),
      StructField("ip", StringType),
      StructField("appid", StringType),
      StructField("appname", StringType),
      StructField("uuid", StringType),
      StructField("device", StringType),
      StructField("client", IntegerType),
      StructField("osversion", StringType),
      StructField("density", StringType),
      StructField("pw", IntegerType),
      StructField("ph", IntegerType),
      StructField("long", StringType),
      StructField("lat", StringType),
      StructField("provincename", StringType),
      StructField("cityname", StringType),
      StructField("ispid", IntegerType),
      StructField("ispname", StringType),
      StructField("networkmannerid", IntegerType),
      StructField("networkmannername", StringType),
      StructField("iseffective", IntegerType),
      StructField("isbilling", IntegerType),
      StructField("adspacetype", IntegerType),
      StructField("adspacetypename", StringType),
      StructField("devicetype", IntegerType),
      StructField("processnode", IntegerType),
      StructField("apptype", IntegerType),
      StructField("district", StringType),
      StructField("paymode", IntegerType),
      StructField("isbid", IntegerType),
      StructField("bidprice", DoubleType),
      StructField("winprice", DoubleType),
      StructField("iswin", IntegerType),
      StructField("cur", StringType),
      StructField("rate", DoubleType),
      StructField("cnywinprice", DoubleType),
      StructField("imei", StringType),
      StructField("mac", StringType),
      StructField("idfa", StringType),
      StructField("openudid", StringType),
      StructField("androidid", StringType),
      StructField("rtbprovince", StringType),
      StructField("rtbcity", StringType),
      StructField("rtbdistrict", StringType),
      StructField("rtbstreet", StringType),
      StructField("storeurl", StringType),
      StructField("realip", StringType),
      StructField("isqualityapp", IntegerType),
      StructField("bidfloor", DoubleType),
      StructField("aw", IntegerType),
      StructField("ah", IntegerType),
      StructField("imeimd5", StringType),
      StructField("macmd5", StringType),
      StructField("idfamd5", StringType),
      StructField("openudidmd5", StringType),
      StructField("androididmd5", StringType),
      StructField("imeisha1", StringType),
      StructField("macsha1", StringType),
      StructField("idfasha1", StringType),
      StructField("openudidsha1", StringType),
      StructField("androididsha1", StringType),
      StructField("uuidunknow", StringType),
      StructField("userid", StringType),
      StructField("iptype", IntegerType),
      StructField("initbidprice", DoubleType),
      StructField("adpayment", DoubleType),
      StructField("agentrate", DoubleType),
      StructField("lrate", DoubleType),
      StructField("adxrate", DoubleType),
      StructField("title", StringType),
      StructField("keywords", StringType),
      StructField("tagid", StringType),
      StructField("callbackdate", StringType),
      StructField("channelid", StringType),
      StructField("mediatype", IntegerType)
    ))
      //导入包外的隐式转换
      import utils.RichString._
    val row: RDD[Row] = file.filter(t => t.split(",").length == 85).map(t => {
      val arr: Array[String] = t.split(",")
      Row(arr(0),
        arr(1).parseInt,
        arr(2).parseInt,
        arr(3).parseInt,
        arr(4).parseInt,
        arr(5),
        arr(6),
        arr(7).parseInt,
        arr(8).parseInt,
        arr(9).parseDouble,
        arr(10).parseDouble,
        arr(11),
        arr(12),
        arr(13),
        arr(14),
        arr(15),
        arr(16),
        arr(17).parseInt,
        arr(18),
        arr(19),
        arr(20).parseInt,
        arr(21).parseInt,
        arr(22),
        arr(23),
        arr(24),
        arr(25),
        arr(26).parseInt,
        arr(27),
        arr(28).parseInt,
        arr(29),
        arr(30).parseInt,
        arr(31).parseInt,
        arr(32).parseInt,
        arr(33),
        arr(34).parseInt,
        arr(35).parseInt,
        arr(36).parseInt,
        arr(37),
        arr(38).parseInt,
        arr(39).parseInt,
        arr(40).parseDouble,
        arr(41).parseDouble,
        arr(42).parseInt,
        arr(43),
        arr(44).parseDouble,
        arr(45).parseDouble,
        arr(46),
        arr(47),
        arr(48),
        arr(49),
        arr(50),
        arr(51),
        arr(52),
        arr(53),
        arr(54),
        arr(55),
        arr(56),
        arr(57).parseInt,
        arr(58).parseDouble,
        arr(59).parseInt,
        arr(60).parseInt,
        arr(61),
        arr(62),
        arr(63),
        arr(64),
        arr(65),
        arr(66),
        arr(67),
        arr(68),
        arr(69),
        arr(70),
        arr(71),
        arr(72),
        arr(73).parseInt,
        arr(74).parseDouble,
        arr(75).parseDouble,
        arr(76).parseDouble,
        arr(77).parseDouble,
        arr(78).parseDouble,
        arr(79),
        arr(80),
        arr(81),
        arr(82),
        arr(83),
        arr(84).parseInt)

    })
    // 自定义DataFrame 要传入一个 RDD[row] ---代表数据内容   StructType--代表数据列表信息
    val createDataFrame: DataFrame = sqlcontext.createDataFrame(row,schema)
    //可以 在write后面调用 partitionBy()  参数中传入 划分区的标准
    // 同时还可以在 write 前面调用 coalesce(传入分区的数量)
    //createDataFrame.coalesce(4).write.partitionBy().parquet("E:\\大数据资料\\project2\\parquet_plus")
    ssc.stop()
  }
}
